package com.spider.ask120.webmagic;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jdiy.core.Rs;

import com.spider.ask120.service.ChannelService;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 健康经验 频道
 * @author Jxw
 *
 */
public class JingyanCatalog  implements PageProcessor{
	static String jkjyUrl = "http://m.120ask.com/jingyan/";
	static String listUrl = "http://m.120ask.com/jingyan/list/\\w+"; 
	

	public static void main(String[] args) {
		Spider.create(new JingyanCatalog()).addUrl("http://m.120ask.com/jingyan/").thread(1).run(); 
		
    }

	public void process(Page page) { 
		//进入频道页,获取各病种列表页URL
		if(page.getUrl().regex(jkjyUrl+"$").match()) {
			//保存频道 
			Rs channel = ChannelService.saveChannel("jingyan", "健康经验", "0");
			if(channel != null ){ 
				Map<String, Object> extras = new HashMap<String, Object>();
				extras.put("parent_id", channel.get("id"));
				
				//获取到话题页列表URL 
				List<String> huati = page.getHtml().links().regex(listUrl).all(); 
				for(String url : huati){ 
					Request request = new Request(url);
					request.setExtras(extras);
					page.addTargetRequest(request);
				}
			}
			
		 
		}else if(page.getUrl().regex(listUrl).match()) {//不包括.html后缀  //+"$" 
			String parent_id = page.getRequest().getExtra("parent_id").toString(); 
			String channelName = page.getHtml().$("#classname","text").get();
			//从URL中获取当前目录ID 
			String path = page.getUrl().toString();
			String str_id = "";
			if (path.indexOf(".html") > 0) {
				str_id = path.substring(path.indexOf("list/")+5, path.indexOf(".html"));
			} else {
				str_id = path.substring(path.indexOf("list/")+5);
			} 
			 
			//保存频道、栏目数据
			Rs channel = ChannelService.saveChannel(str_id, channelName, parent_id);
			if(channel != null ){ 
				//判断该页面中如果还有 下一级 栏目， 则继续获取连接
				if(page.getHtml().$(".s_content").match()){
					List<String> huati = page.getHtml().$(".s_content").links().regex(listUrl).all(); 

					Map<String, Object> extras = new HashMap<String, Object>();
					extras.put("parent_id", channel.get("id")); 
					
					for(String url : huati){ 
						Request request = new Request(url);
						request.setExtras(extras);
						page.addTargetRequest(request);
					}
				}  
			}
			 
			 
		} 
		 

	}
  

	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1);
	public Site getSite() {  
		return site;
	}
}
